# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapyuniversal.items import *
from scrapyuniversal.loaders import ChinaLoader


class ChinaSpider(CrawlSpider):
    name = 'china'
    allowed_domains = ['tech.china.com']
    start_urls = ['http://tech.china.com/articles/']

    rules = (
        Rule(LinkExtractor(allow='article.*.html',
                           restrict_xpaths='//div[@id="rank-defList"]//div[@class="wntjItem item_defaultView clearfix"]'),
             callback='parse_item'),
        Rule(LinkExtractor(restrict_xpaths='//*[@id="rank-defList"]//div[@class="pages"]//a[13]'))
    )

    def parse_item(self, response):
        loadder = ChinaLoader(item=NewsItem(), response=response)
        loadder.add_xpath('title','//h1[@id="chan_newsTitle"]/text()')
        loadder.add_value('url',response.url)
        loadder.add_xpath('text','//div[@id="chan_newsDetail"]//text()')
        loadder.add_xpath('datetime','//div[@class="chan_newsInfo_source"]/span[@class="time"]/text()')
        loadder.add_xpath('source','//div[@class="chan_newsInfo_source"]/span[@class="source"]/text()')
        loadder.add_value('website','中华网')
        yield loadder.load_item()
        # item = NewsItem()
        # item['title'] = response.xpath('//h1[@id="chan_newsTitle"]/text()').extract_first()
        # item['url'] = response.url
        # item['text'] = ''.join(response.xpath('//div[@id="chan_newsDetail"]//text()').extract()).strip()
        # item['datetime'] = response.xpath('//div[@class="chan_newsInfo_source"]/span[@class="time"]/text()').extract_first()
        # item['source'] = response.xpath('//div[@class="chan_newsInfo_source"]/span[@class="source"]/text()').extract_first()
        # item['website'] = '中华网'
        # yield item
